const fs = require("fs");
const path = require("path");
const iconv = require("iconv-lite");
const { cleanDatEnglishText } = require("../src/utils/datTextCleaner");

// 读取实际的DAT文件
const datFilePath = path.join(__dirname, "../input/dat/15.dat");
const buffer = fs.readFileSync(datFilePath);
const data = iconv.decode(buffer, "gbk");

console.log("原始DAT文件内容预览:");
console.log(data.substring(0, 200) + "...\n");

// 分割为各个条目
const entries = data.split("*^").filter(entry => entry.trim() !== "");

console.log(`总共找到 ${entries.length} 个条目\n`);

// 测试前几个条目
for (let i = 0; i < Math.min(5, entries.length); i++) {
  const entry = entries[i];
  if (entry.trim() === "") continue;

  // 提取英语部分（在第一个 # 之前）
  const parts = entry.split("#");
  if (parts.length > 0) {
    const englishPart = parts[0];
    const cleanedText = cleanDatEnglishText(englishPart);

    console.log(`条目 ${i + 1}:`);
    console.log(`  原始: ${englishPart}`);
    console.log(`  清理后: ${cleanedText}`);
    console.log("");
  }
}

// 特别关注一些有重复词问题的条目
console.log("重点测试包含重复词的条目:");
const problematicEntries = [
  "Please$please$$$$$ ^give$give$$$$$ ^our$our$$$$$ ^claim$claim$$$$$ ^your$your$$$$$ ^favorable$favorable$$$$$^ ^consideration$consideration$$$$$^.",
  "as soon as possible$as soon as possible$$$$$ ^.",
  "in view of$in view of$$$$$ ^our$our$$$$$ ^friendly$friendly$$$$$ ^business$business$$$$$ ^relations$relation$$$$$ ^we$we$$$$$ ^are$are$$$$$ ^prepared$prepared$$$$$ ^to$to$$$$$ ^meet$meet$$$$$ ^your$your$$$$$ ^claim$claim$$$$$ ^for$for$$$$$ ^the$the$$$$$ 25 ^tons$ton$$$$$ ^shortage$shortage$$$$$ ^weight$weight$$$$$^.",
  "due to$due to$$$$$ ^omission$omission$$$$$^.",
];

problematicEntries.forEach((entry, index) => {
  const cleanedText = cleanDatEnglishText(entry);
  console.log(`问题条目 ${index + 1}:`);
  console.log(`  原始: ${entry}`);
  console.log(`  清理后: ${cleanedText}`);
  console.log("");
});
